#!/usr/bin/env python

"""scrap_htmlfiles.py
Scraping html files with htmldata
"""

__author__  = "Daniel Berns"
__app__     = "scrap_htmlfiles"
__version__ = "0.0.2"
__date__    = "20080611"

import os
import re
from htmldata import *

def scrap_html(file_to_scrap, results):
    """
    Scrap html from a saved html page
    """
    print file_to_scrap
    source = open(file_to_scrap, 'r')
    if source == None: return None
    content = source.read()
    source.close()
    if content == None: return None
    results.add(content)

def scrap_them_all(input_dir, filelist, results):
    """
    Scrap the whole set of html files
    """
    source = open(''.join([input_dir, os.sep, filelist]), 'r')
    if source == None: return None
    for item in source:
        line = item[:-1]
        scrap_html(''.join([input_dir, os.sep, line]), results)

class attr_info:
   def __init__(self):
       self.counter = 0
       self.setofvalues = set()

   def add(self, value):
       self.counter = self.counter + 1
       if value:
          self.setofvalues.add(value)
       else:
          self.setofvalues.add('unknown')

   def dump(self, output):
       output.write('\t\tcounter %d\n' % self.counter)
       for value in self.setofvalues:
          if value:
             output.write('\t\t%s\n' % value)

class tag_info:
   def __init__(self):
       self.counter = 0
       self.attr_info_map = {}

   def add(self, attrs):
       if attrs:
          self.counter = self.counter + 1;
          for key in attrs.keys():
              value = attrs[key]

              if self.attr_info_map.has_key(key) == False:
                  self.attr_info_map[key] = attr_info()

              self.attr_info_map[key].add(value)

   def dump(self, output):
       output.write('\tcounter : %d\n' % self.counter)
       for key in self.attr_info_map.keys():
           output.write('\tattr: %s\n' % key)
           self.attr_info_map[key].dump(output)

class HTMLResults:
   """
   Callback for saving results
   """

   def __init__(self, output_dir, text_filename, tags_filename, stats_filename):
       self.text = open(''.join([output_dir, os.sep, text_filename]), 'w')
       self.tags = open(''.join([output_dir, os.sep, tags_filename]), 'w')
       self.stats = open(''.join([output_dir, os.sep, stats_filename]), 'w')
       self.tag_info_map = {}
       self.last_tag = None
   
   def add(self, content):
       """
       Sanitize text, extract words, collects and counts tags and attributes
       """
       data = tagextract(content)
       if data == None: return None

       for a0 in data:
           if isisntance(a0, str)

             if self.last_tag != 'script':
                if len(a0) > 0:
                   if a0.startswith('[[') == False:
                      a1 = a0.replace("(", " ").replace(")", " ").replace("[", " ").replace("]", " ")
                      a2 = a1.replace("{", " ").replace("}", " ").replace("[", " ").replace("]", " ")
                      a3 = a2.replace("'", " ").replace('"', " ").replace("?", " ").replace("^", " ")
                      a4 = a3.replace(",", " ").replace(".", " ").replace(";", " ").replace(":", " ")
                      a5 = a4.replace("+", " ").replace("-", " ").replace("*", " ").replace("/", " ")
                      a6 = a5.replace("$", " ").replace("!", " ").replace("&", " ").replace("#160;", " ")
                      seq = a6.split()
                      for item in seq:
                          self.text.write("%s\n" % item)
                self.last_tag = None
           else:
             (name, attrs) = a0
             self.last_tag = name
         
             if (len(name) > 0) & (name.startswith('!--') == False):
                if self.tag_info_map.has_key(name):
                   self.tag_info_map[name].add(attrs)
                else:
                   self.tag_info_map[name] = tag_info()
                   self.tag_info_map[name].add(attrs)

                self.tags.write("tag name: %s\n" % name)
                if attrs:
                   self.tags.write("attrs\n")
                   for item in attrs:
                       self.tags.write("  %s\n" % repr(item))

   def close(self):
       """
       Close files to avoid data loss
       """
       self.text.close()
       self.tags.close()

       for item in self.tag_info_map.keys():
           self.stats.write('tag : %s\n' % item)
           self.tag_info_map[item].dump(self.stats)

       self.stats.close()
        
def scrap(output_dir, words, tags, stats, input_dir, filenames):
   results = HTMLResults(output_dir, words, tags, stats);
   scrap_them_all(input_dir, filenames, results)
   results.close()
